#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 
# Copyright (c) 2017 Baidu.com, Inc. All Rights Reserved
# 

"""
File: utils.py
Author: zhangyang(zhangyang40@baidu.com)
Date: 2017/10/20 10:24
"""
import argparse
from sklearn.externals.joblib import load, dump
import pandas as pd
import numpy as np

# config
# where we can find training, test, and sampleSubmission.csv
raw_data_path = 'D:/Data/'
# where we store results -- require about 130GB
tmp_data_path = 'D:/Data/temp/'

try:
    params = load(tmp_data_path + '_params.joblib_dat')
    sample_pct = params['pct']
    tvh = params['tvh']
except:
    pass


def print_help():
    print "usage: python utils -set_params [tvh=Y|N], [sample_pct]"
    print "for example: python utils -set_params N 0.05"


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-set_params", dest="set_params", choices=['Y', 'N'])
    parser.add_argument("-sample_pct", dest="sample_pct", type=float)
    args = parser.parse_args()
    tvh = args.set_params
    sample_pct = args.sample_pct
    dump({'pct': sample_pct, 'tvh': tvh}, tmp_data_path + '_params.joblib_dat')


def calcTVTransform(df, vn, vn_y, cred_k, filter_train, mean0=None):
    print "*" * 100
    print 'vn', vn
    if mean0 is None:
        mean0 = df.ix[filter_train, vn_y].mean()
        print "mean0:", mean0
    else:
        mean0 = mean0[~filter_train]

    df['_key1'] = df[vn].astype('category').values.codes
    df_yt = df.ix[filter_train, ['_key1', vn_y]]
    # df_y.set_index([')key1'])
    grp1 = df_yt.groupby(['_key1'])
    sum1 = grp1[vn_y].aggregate(np.sum)
    print sum1
    cnt1 = grp1[vn_y].aggregate(np.size)
    print cnt1
    vn_sum = 'sum_' + vn
    vn_cnt = 'cnt_' + vn
    v_codes = df.ix[~filter_train, '_key1']
    print v_codes
    _sum = sum1[v_codes].values
    _cnt = cnt1[v_codes].values
    _cnt[np.isnan(_sum)] = 0
    _sum[np.isnan(_sum)] = 0

    r = {}
    r['exp'] = (_sum + cred_k * mean0) / (_cnt + cred_k)
    r['cnt'] = _cnt
    return r


def calc_exptv(t0, vn_list, last_day_only=False, add_count=False):
    t0a = t0.ix[:, ['day', 'click']].copy()
    day_exps = {}

    for vn in vn_list:
        if vn == 'dev_id_ip':
            t0a[vn] = pd.Series(np.add(t0.device_id.values, t0.device_ip.values)).astype('category').values.codes
        elif vn == 'dev_ip_aw':
            t0a[vn] = pd.Series(np.add(t0.device_ip.values, t0.app_or_web.astype('string').values)).astype(
                'category').values.codes
        elif vn == 'C14_aw':
            t0a[vn] = pd.Series(np.add(t0.C14.astype('string').values, t0.app_or_web.astype('string').values)).astype(
                'category').values.codes
        elif vn == 'C17_aw':
            t0a[vn] = pd.Series(np.add(t0.C17.astype('string').values, t0.app_or_web.astype('string').values)).astype(
                'category').values.codes
        elif vn == 'C21_aw':
            t0a[vn] = pd.Series(np.add(t0.C21.astype('string').values, t0.app_or_web.astype('string').values)).astype(
                'category').values.codes
        elif vn == 'as_domain':
            t0a[vn] = pd.Series(np.add(t0.app_domain.values, t0.site_domain.values)).astype('category').values.codes
        elif vn == 'site_app_id':
            t0a[vn] = pd.Series(np.add(t0.site_id.values, t0.app_id.values)).astype('category').values.codes
        elif vn == 'app_model':
            t0a[vn] = pd.Series(np.add(t0.app_id.values, t0.device_model.values)).astype('category').values.codes
        elif vn == 'app_site_model':
            t0a[vn] = pd.Series(np.add(t0.app_id.values, np.add(t0.site_id.values, t0.device_model.values))).astype(
                'category').values.codes
        elif vn == 'site_model':
            t0a[vn] = pd.Series(np.add(t0.site_id.values, t0.device_model.values)).astype('category').values.codes
        elif vn == 'app_site':
            t0a[vn] = pd.Series(np.add(t0.app_id.values, t0.site_id.values)).astype('category').values.codes
        elif vn == 'site_ip':
            t0a[vn] = pd.Series(np.add(t0.site_id.values, t0.device_ip.values)).astype('category').values.codes
        elif vn == 'app_ip':
            t0a[vn] = pd.Series(np.add(t0.site_id.values, t0.device_ip.values)).astype('category').values.codes
        elif vn == 'site_id_domain':
            t0a[vn] = pd.Series(np.add(t0.site_id.values, t0.site_domain.values)).astype('category').values.codes
        elif vn == 'site_hour':
            t0a[vn] = pd.Series(np.add(t0.site_domain.values, (t0.hour.values % 100).astype('string'))).astype(
                'category').values.codes
        else:
            t0a[vn] = t0[vn]

        for day_v in xrange(22, 32):
            cred_k = 10
            if day_v not in day_exps:
                day_exps[day_v] = {}

            vn_key = vn

            import time
            _tstart = time.time()

            day1 = 20
            if last_day_only:
                day1 = day_v - 2
            filter_t = np.logical_and(t0.day.values > day1, t0.day.values <= day_v)
            vn_key = vn
            t1 = t0a.ix[filter_t, :].copy()
            filter_t2 = np.logical_and(t1.day.values != day_v, t1.day.values < 31)

            if vn == 'app_or_web':
                t = calcTVTransform(t1, vn, 'click', cred_k, filter_t2)
                day_exps[day_v][vn_key] = t
            else:
                if last_day_only:
                    day_exps[day_v][vn_key] = calcTVTransform(t1, vn, 'click', cred_k, filter_t2,
                                                              mean0=t0.expld_app_or_web.values)
                else:
                    day_exps[day_v][vn_key] = calcTVTransform(t1, vn, 'click', cred_k, filter_t2,
                                                              mean0=t0.exptv_app_or_web.values)

                    # print vn, vn_key, " ", day_v, " done in ", time.time() - _tstart
        t0a.drop(vn, inplace=True, axis=1)

    for vn in vn_list:
        vn_key = vn

        vn_exp = 'exptv_' + vn_key
        if last_day_only:
            vn_exp = 'expld_' + vn_key

        t0[vn_exp] = np.zeros(t0.shape[0])
        if add_count:
            t0['cnttv_' + vn_key] = np.zeros(t0.shape[0])
        for day_v in xrange(22, 32):
            # print vn, vn_key, day_v, t0.ix[t0.day.values == day_v, vn_exp].values.size, day_exps[day_v][vn_key][
            #     'exp'].size
            t0.loc[t0.day.values == day_v, vn_exp] = day_exps[day_v][vn_key]['exp']
            if add_count:
                t0.loc[t0.day.values == day_v, 'cnttv_' + vn_key] = day_exps[day_v][vn_key]['cnt']


if __name__ == '__main__':
    main()
